<!DOCTYPE html>
<html lang="zh-CN">

<head>
  <meta charset="UTF-8">
  <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=2">
  <meta name="theme-color" content="#222">
  <meta name="generator" content="Hexo 4.2.1">
  <link rel="apple-touch-icon" sizes="180x180" href="/images/apple-touch-icon-next.png">
  <link rel="icon" type="image/png" sizes="32x32" href="/images/favicon-32x32-next.png">
  <link rel="icon" type="image/png" sizes="16x16" href="/images/favicon-16x16-next.png">
  <link rel="mask-icon" href="/images/safari-pinned-tab.svg" color="#222">
  <link rel="stylesheet" href="/css/main.css">
  <link rel="stylesheet" href="/lib/font-awesome/css/all.min.css">
  <link rel="stylesheet" href="/lib/pace/pace-theme-minimal.min.css">
  <script src="/lib/pace/pace.min.js"></script>
  <script id="hexo-configurations">
    var NexT = window.NexT ||
    {};
    var CONFIG = {
      "hostname": "cuiqingcai.com",
      "root": "/",
      "scheme": "Pisces",
      "version": "7.8.0",
      "exturl": false,
      "sidebar":
      {
        "position": "right",
        "width": 360,
        "display": "post",
        "padding": 18,
        "offset": 12,
        "onmobile": false,
        "widgets": [
          {
            "type": "image",
            "name": "阿布云",
            "enable": false,
            "url": "https://www.abuyun.com/http-proxy/introduce.html",
            "src": "https://qiniu.cuiqingcai.com/88au8.jpg",
            "width": "100%"
      },
          {
            "type": "image",
            "name": "天验",
            "enable": true,
            "url": "https://tutorial.lengyue.video/?coupon=12ef4b1a-a3db-11ea-bb37-0242ac130002_cqx_850",
            "src": "https://qiniu.cuiqingcai.com/bco2a.png",
            "width": "100%"
      },
          {
            "type": "image",
            "name": "华为云",
            "enable": false,
            "url": "https://activity.huaweicloud.com/2020_618_promotion/index.html?bpName=5f9f98a29e2c40b780c1793086f29fe2&bindType=1&salesID=wangyubei",
            "src": "https://qiniu.cuiqingcai.com/y42ik.jpg",
            "width": "100%"
      },
          {
            "type": "image",
            "name": "张小鸡",
            "enable": false,
            "url": "http://www.zxiaoji.com/",
            "src": "https://qiniu.cuiqingcai.com/fm72f.png",
            "width": "100%"
      },
          {
            "type": "image",
            "name": "Luminati",
            "src": "https://qiniu.cuiqingcai.com/ikkq9.jpg",
            "url": "https://luminati-china.io/?affiliate=ref_5fbbaaa9647883f5c6f77095",
            "width": "100%",
            "enable": false
      },
          {
            "type": "image",
            "name": "IPIDEA",
            "url": "http://www.ipidea.net/?utm-source=cqc&utm-keyword=?cqc",
            "src": "https://qiniu.cuiqingcai.com/0ywun.png",
            "width": "100%",
            "enable": true
      },
          {
            "type": "tags",
            "name": "标签云",
            "enable": true
      },
          {
            "type": "categories",
            "name": "分类",
            "enable": true
      },
          {
            "type": "friends",
            "name": "友情链接",
            "enable": true
      },
          {
            "type": "hot",
            "name": "猜你喜欢",
            "enable": true
      }]
      },
      "copycode":
      {
        "enable": true,
        "show_result": true,
        "style": "mac"
      },
      "back2top":
      {
        "enable": true,
        "sidebar": false,
        "scrollpercent": true
      },
      "bookmark":
      {
        "enable": false,
        "color": "#222",
        "save": "auto"
      },
      "fancybox": false,
      "mediumzoom": false,
      "lazyload": false,
      "pangu": true,
      "comments":
      {
        "style": "tabs",
        "active": "gitalk",
        "storage": true,
        "lazyload": false,
        "nav": null,
        "activeClass": "gitalk"
      },
      "algolia":
      {
        "hits":
        {
          "per_page": 10
        },
        "labels":
        {
          "input_placeholder": "Search for Posts",
          "hits_empty": "We didn't find any results for the search: ${query}",
          "hits_stats": "${hits} results found in ${time} ms"
        }
      },
      "localsearch":
      {
        "enable": true,
        "trigger": "auto",
        "top_n_per_article": 10,
        "unescape": false,
        "preload": false
      },
      "motion":
      {
        "enable": false,
        "async": false,
        "transition":
        {
          "post_block": "bounceDownIn",
          "post_header": "slideDownIn",
          "post_body": "slideDownIn",
          "coll_header": "slideLeftIn",
          "sidebar": "slideUpIn"
        }
      },
      "path": "search.xml"
    };

  </script>
  <meta name="description" content="这博文写得我懒癌犯了，最后的那个章节内容排序，我没有实验是否是正确的，不过这只是个教大家用Scrapy的教程，正确与否并不重要··· 如果不正确，记得留言；等我懒癌过了，我再改改······ 还有其它的问题也是一样··· ，把问题留言下； 等我懒癌过了·· 我改回来！嗯！是等我懒癌结束了，再改。 前面几篇博文，给大家从头到尾做了一个比较高效的爬虫，从这篇起来说说Python的爬虫框架Scrapy；">
  <meta property="og:type" content="article">
  <meta property="og:title" content="小白进阶之Scrapy第一篇">
  <meta property="og:url" content="https://cuiqingcai.com/3472.html">
  <meta property="og:site_name" content="静觅">
  <meta property="og:description" content="这博文写得我懒癌犯了，最后的那个章节内容排序，我没有实验是否是正确的，不过这只是个教大家用Scrapy的教程，正确与否并不重要··· 如果不正确，记得留言；等我懒癌过了，我再改改······ 还有其它的问题也是一样··· ，把问题留言下； 等我懒癌过了·· 我改回来！嗯！是等我懒癌结束了，再改。 前面几篇博文，给大家从头到尾做了一个比较高效的爬虫，从这篇起来说说Python的爬虫框架Scrapy；">
  <meta property="og:locale" content="zh_CN">
  <meta property="og:image" content="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/10/9555112.jpg">
  <meta property="og:image" content="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/11/%E5%AE%89%E8%A3%85Scrapy.gif">
  <meta property="og:image" content="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/11/%E5%88%9B%E5%BB%BA%E9%A1%B9%E7%9B%AE.gif">
  <meta property="og:image" content="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/11/%E5%90%84%E4%B8%AA%E6%96%87%E4%BB%B6%E7%9A%84%E4%BD%9C%E7%94%A8.png">
  <meta property="og:image" content="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/11/%E5%BF%AB%E6%8D%B7%E5%90%AF%E5%8A%A8.png">
  <meta property="og:image" content="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/12/scrapy_architecture.png">
  <meta property="og:image" content="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/10/QQ%E5%9B%BE%E7%89%8720161022193315.gif">
  <meta property="og:image" content="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/12/%E8%AE%BE%E7%BD%AEsetting01.png">
  <meta property="og:image" content="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/12/Scrapy01.png">
  <meta property="og:image" content="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/12/Scrapy02.png">
  <meta property="og:image" content="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/12/Scrapy22.png">
  <meta property="og:image" content="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/11/Spider%E7%BC%96%E5%86%9903.png">
  <meta property="og:image" content="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/11/%E5%88%86%E6%9E%90%E7%BD%91%E9%A1%B52.png">
  <meta property="og:image" content="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/11/%E5%88%86%E6%9E%90%E7%BD%91%E9%A1%B501.png">
  <meta property="og:image" content="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/12/Scrapy20.png">
  <meta property="og:image" content="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/12/Scrapy07.png">
  <meta property="og:image" content="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/12/Scrapy09.png">
  <meta property="og:image" content="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/12/Scrapy10.png">
  <meta property="og:image" content="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/12/Scrapy11.png">
  <meta property="og:image" content="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/12/Scrapy12.png">
  <meta property="og:image" content="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/12/MySQL-setting.png">
  <meta property="og:image" content="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/12/sql01.png">
  <meta property="og:image" content="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/12/sql01-1.png">
  <meta property="og:image" content="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/12/pipeline02.png">
  <meta property="og:image" content="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/12/setting02.png">
  <meta property="og:image" content="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/12/scrapy15.png">
  <meta property="og:image" content="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/12/scrapy17.png">
  <meta property="og:image" content="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/12/scrapy16.png">
  <meta property="og:image" content="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/12/scrapy18.png">
  <meta property="og:image" content="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/12/Scrapy13.png">
  <meta property="og:image" content="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/12/Scrapy14.png">
  <meta property="og:image" content="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/12/scrapy21.png">
  <meta property="og:image" content="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/12/Scrapy16.png">
  <meta property="article:published_time" content="2016-12-07T08:44:48.000Z">
  <meta property="article:modified_time" content="2021-12-18T13:11:11.569Z">
  <meta property="article:author" content="崔庆才">
  <meta property="article:tag" content="崔庆才">
  <meta property="article:tag" content="静觅">
  <meta property="article:tag" content="PHP">
  <meta property="article:tag" content="Java">
  <meta property="article:tag" content="Python">
  <meta property="article:tag" content="Spider">
  <meta property="article:tag" content="爬虫">
  <meta property="article:tag" content="Web">
  <meta property="article:tag" content="Kubernetes">
  <meta property="article:tag" content="深度学习">
  <meta property="article:tag" content="机器学习">
  <meta property="article:tag" content="数据分析">
  <meta property="article:tag" content="网络">
  <meta property="article:tag" content="IT">
  <meta property="article:tag" content="技术">
  <meta property="article:tag" content="博客">
  <meta name="twitter:card" content="summary">
  <meta name="twitter:image" content="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/10/9555112.jpg">
  <link rel="canonical" href="https://cuiqingcai.com/3472.html">
  <script id="page-configurations">
    // https://hexo.io/docs/variables.html
    CONFIG.page = {
      sidebar: "",
      isHome: false,
      isPost: true,
      lang: 'zh-CN'
    };

  </script>
  <title>小白进阶之Scrapy第一篇 | 静觅</title>
  <meta name="google-site-verification" content="p_bIcnvirkFzG2dYKuNDivKD8-STet5W7D-01woA2fc" />
  <noscript>
    <style>
      .use-motion .brand,
      .use-motion .menu-item,
      .sidebar-inner,
      .use-motion .post-block,
      .use-motion .pagination,
      .use-motion .comments,
      .use-motion .post-header,
      .use-motion .post-body,
      .use-motion .collection-header
      {
        opacity: initial;
      }

      .use-motion .site-title,
      .use-motion .site-subtitle
      {
        opacity: initial;
        top: initial;
      }

      .use-motion .logo-line-before i
      {
        left: initial;
      }

      .use-motion .logo-line-after i
      {
        right: initial;
      }

    </style>
  </noscript>
  <link rel="alternate" href="/atom.xml" title="静觅" type="application/atom+xml">
</head>

<body itemscope itemtype="http://schema.org/WebPage">
  <div class="container">
    <div class="headband"></div>
    <header class="header" itemscope itemtype="http://schema.org/WPHeader">
      <div class="header-inner">
        <div class="site-brand-container">
          <div class="site-nav-toggle">
            <div class="toggle" aria-label="切换导航栏">
              <span class="toggle-line toggle-line-first"></span>
              <span class="toggle-line toggle-line-middle"></span>
              <span class="toggle-line toggle-line-last"></span>
            </div>
          </div>
          <div class="site-meta">
            <a href="/" class="brand" rel="start">
              <span class="logo-line-before"><i></i></span>
              <h1 class="site-title">静觅 <span class="site-subtitle"> 崔庆才的个人站点 </span>
              </h1>
              <span class="logo-line-after"><i></i></span>
            </a>
          </div>
          <div class="site-nav-right">
            <div class="toggle popup-trigger">
              <i class="fa fa-search fa-fw fa-lg"></i>
            </div>
          </div>
        </div>
        <nav class="site-nav">
          <ul id="menu" class="main-menu menu">
            <li class="menu-item menu-item-home">
              <a href="/" rel="section">首页</a>
            </li>
            <li class="menu-item menu-item-archives">
              <a href="/archives/" rel="section">文章列表</a>
            </li>
            <li class="menu-item menu-item-tags">
              <a href="/tags/" rel="section">文章标签</a>
            </li>
            <li class="menu-item menu-item-categories">
              <a href="/categories/" rel="section">文章分类</a>
            </li>
            <li class="menu-item menu-item-about">
              <a href="/about/" rel="section">关于博主</a>
            </li>
            <li class="menu-item menu-item-message">
              <a href="/message/" rel="section">给我留言</a>
            </li>
            <li class="menu-item menu-item-search">
              <a role="button" class="popup-trigger">搜索 </a>
            </li>
          </ul>
        </nav>
        <div class="search-pop-overlay">
          <div class="popup search-popup">
            <div class="search-header">
              <span class="search-icon">
                <i class="fa fa-search"></i>
              </span>
              <div class="search-input-container">
                <input autocomplete="off" autocapitalize="off" placeholder="搜索..." spellcheck="false" type="search" class="search-input">
              </div>
              <span class="popup-btn-close">
                <i class="fa fa-times-circle"></i>
              </span>
            </div>
            <div id="search-result">
              <div id="no-result">
                <i class="fa fa-spinner fa-pulse fa-5x fa-fw"></i>
              </div>
            </div>
          </div>
        </div>
      </div>
    </header>
    <div class="back-to-top">
      <i class="fa fa-arrow-up"></i>
      <span>0%</span>
    </div>
    <div class="reading-progress-bar"></div>
    <main class="main">
      <div class="main-inner">
        <div class="content-wrap">
          <div class="content post posts-expand">
            <article itemscope itemtype="http://schema.org/Article" class="post-block single" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/3472.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h1 class="post-title" itemprop="name headline"> 小白进阶之Scrapy第一篇 </h1>
                <div class="post-meta">
                  <span class="post-meta-item">
                    <span class="post-meta-item-icon">
                      <i class="far fa-user"></i>
                    </span>
                    <span class="post-meta-item-text">作者</span>
                    <span><a href="/authors/哎哟卧槽" class="author" itemprop="url" rel="index">哎哟卧槽</a></span>
                  </span>
                  <span class="post-meta-item">
                    <span class="post-meta-item-icon">
                      <i class="far fa-calendar"></i>
                    </span>
                    <span class="post-meta-item-text">发表于</span>
                    <time title="创建时间：2016-12-07 16:44:48" itemprop="dateCreated datePublished" datetime="2016-12-07T16:44:48+08:00">2016-12-07</time>
                  </span>
                  <span class="post-meta-item">
                    <span class="post-meta-item-icon">
                      <i class="far fa-folder"></i>
                    </span>
                    <span class="post-meta-item-text">分类于</span>
                    <span itemprop="about" itemscope itemtype="http://schema.org/Thing">
                      <a href="/categories/Python/" itemprop="url" rel="index"><span itemprop="name">Python</span></a>
                    </span>
                  </span>
                  <span id="/3472.html" class="post-meta-item leancloud_visitors" data-flag-title="小白进阶之Scrapy第一篇" title="阅读次数">
                    <span class="post-meta-item-icon">
                      <i class="fa fa-eye"></i>
                    </span>
                    <span class="post-meta-item-text">阅读次数：</span>
                    <span class="leancloud-visitors-count"></span>
                  </span>
                  <span class="post-meta-item" title="本文字数">
                    <span class="post-meta-item-icon">
                      <i class="far fa-file-word"></i>
                    </span>
                    <span class="post-meta-item-text">本文字数：</span>
                    <span>8.7k</span>
                  </span>
                  <span class="post-meta-item" title="阅读时长">
                    <span class="post-meta-item-icon">
                      <i class="far fa-clock"></i>
                    </span>
                    <span class="post-meta-item-text">阅读时长 &asymp;</span>
                    <span>8 分钟</span>
                  </span>
                </div>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="advertisements">
                  <div class="item">
                    <a href="http://i0k.cn/4UUsd" target="_blank">
                      <img src="https://qiniu.cuiqingcai.com/dsdhf.jpg">
                    </a>
                  </div>
                </div>
                <p>这博文写得我懒癌犯了，最后的那个章节内容排序，我没有实验是否是正确的，不过这只是个教大家用Scrapy的教程，正确与否并不重要··· 如果不正确，记得留言；等我懒癌过了，我再改改······ 还有其它的问题也是一样··· ，把问题留言下； 等我懒癌过了·· 我改回来！嗯！是等我懒癌结束了，再改。 前面几篇博文，给大家从头到尾做了一个比较高效的爬虫，从这篇起来说说Python的爬虫框架Scrapy； 至于为什么要说框架呢？因为啊，框架可以帮我们处理一部分事情，比如下载模块不用我们自己写了，我们只需专注于提取数据就好了； 最重要的一点啊！框架使用了异步的模式;可以加快我们的下载速度，而不用自己去实现异步框架；毕竟实现异步爬虫是一件比较麻烦的事情。 不过啊！反爬虫这个坎还是要我们自己迈过去啊！这是后话，以后再说。我们先来让Scrapy能跑起来，并提取出我们需要的数据，再解决其它问题。 官方文档在这儿：点我 <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/10/9555112.jpg" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/10/9555112.jpg" alt="9555112"></a> <strong>环境搭建：</strong> 关于这一点，对在Windows环境下使用的小伙伴来说，请务必使用我之前提到的 Anaconda 这个Python的发行版本，不然光环境的各种报错就能消磨掉你所有的学习兴趣！ <strong>下载地址在这儿：<a href="http://pan.baidu.com/s/1pLgySav" target="_blank" rel="noopener">http://pan.baidu.com/s/1pLgySav</a></strong> 安装完成之后，在cmd中执行：conda install Scrapy (如果需要使用特定版本，请在Scrapy后面加上 ==XXXX XXXX代表你需要的版本号) 下面是安装示意图： <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/11/安装Scrapy.gif" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/11/安装Scrapy.gif" alt="安装Scrapy"></a> So Easy@@！环境搭建完成！是不是超简单？全程无痛啊！ 下面开始踏上新的征程！Go Go Go！！ 使用Scrapy第一步：创建项目；CMD进入你需要放置项目的目录 输入：</p>
                <figure class="highlight mipsasm">
                  <table>
                    <tr>
                      <td class="gutter">
                        <pre><span class="line">1</span><br></pre>
                      </td>
                      <td class="code">
                        <pre><span class="line"><span class="keyword">scrapy </span>startproject XXXXX             XXXXX代表你项目的名字</span><br></pre>
                      </td>
                    </tr>
                  </table>
                </figure>
                <p><a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/11/创建项目.gif" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/11/创建项目.gif" alt="创建项目"></a> OK项目创建完成。现在可以开始我们的爬取之旅了！ 下面是目录中各个文件的作用 <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/11/各个文件的作用.png" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/11/各个文件的作用.png" alt="各个文件的作用"></a> 好了，目录我们认识完了，在开始之前给大家一个小技巧，Scrapy默认是不能在IDE中调试的，我们在根目录中新建一个py文件叫：entrypoint.py；在里面写入以下内容：</p>
                <figure class="highlight smali">
                  <table>
                    <tr>
                      <td class="gutter">
                        <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                      </td>
                      <td class="code">
                        <pre><span class="line">from scrapy.cmdline import execute</span><br><span class="line">execute(['scrapy', 'crawl', 'dingdian'])</span><br></pre>
                      </td>
                    </tr>
                  </table>
                </figure>
                <p><strong>注意！第二行中代码中的前两个参数是不变的，第三个参数请使用自己的spider的名字。稍后我会讲到！！</strong> 现在整个目录看起来是这样： <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/11/快捷启动.png" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/11/快捷启动.png" alt="快捷启动"></a> 基础工作准备完毕！我们来说说基本思路。 上面的准备工作完成之后，我们先不要着急开始工作，毕竟作为一个框架，还是很复杂的；贸然上手 开整，很容易陷入懵逼状态啊！一团浆糊，理不清思路，后面的事情做起来很很麻烦啦！ 我们来看看下面这张图： <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/12/scrapy_architecture.png" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/12/scrapy_architecture.png" alt="scrapy_architecture"></a> 这就是整个Scrapy的架构图了； <strong>Scrapy Engine: 这是引擎，负责Spiders、ItemPipeline、Downloader、Scheduler中间的通讯，信号、数据传递等等！（像不像人的身体？）</strong> <strong>Scheduler(调度器): 它负责接受引擎发送过来的requests请求，并按照一定的方式进行整理排列，入队、并等待Scrapy Engine(引擎)来请求时，交给引擎。</strong> <strong>Downloader（下载器）：负责下载Scrapy Engine(引擎)发送的所有Requests请求，并将其获取到的Responses交还给Scrapy Engine(引擎)，由引擎交给Spiders来处理，</strong> <strong>Spiders：它负责处理所有Responses,从中分析提取数据，获取Item字段需要的数据，并将需要跟进的URL提交给引擎，再次进入Scheduler(调度器)，</strong> <strong>Item Pipeline：它负责处理Spiders中获取到的Item，并进行处理，比如去重，持久化存储（存数据库，写入文件，总之就是保存数据用的）</strong> <strong>Downloader Middlewares（下载中间件）：你可以当作是一个可以自定义扩展下载功能的组件</strong> <strong>Spider Middlewares（Spider中间件）：你可以理解为是一个可以自定扩展和操作引擎和Spiders中间‘通信‘的功能组件（比如进入Spiders的Responses;和从Spiders出去的Requests）</strong> <strong>数据在整个Scrapy的流向：</strong> 程序运行的时候， <strong>引擎：</strong>Hi！Spider, 你要处理哪一个网站？ <strong>Spiders：</strong>我要处理23wx.com <strong>引擎：</strong>你把第一个需要的处理的URL给我吧。 <strong>Spiders：</strong>给你第一个URL是XXXXXXX.com <strong>引擎：</strong>Hi！调度器，我这有request你帮我排序入队一下。 <strong>调度器：</strong>好的，正在处理你等一下。 <strong>引擎：</strong>Hi！调度器，把你处理好的request给我， <strong>调度器：</strong>给你，这是我处理好的request <strong>引擎：</strong>Hi！下载器，你按照下载中间件的设置帮我下载一下这个request <strong>下载器：</strong>好的！给你，这是下载好的东西。（如果失败：不好意思，这个request下载失败，然后<strong>引擎</strong>告诉<strong>调度器</strong>，这个request下载失败了，你记录一下，我们待会儿再下载。） <strong>引擎：</strong>Hi！Spiders，这是下载好的东西，并且已经按照Spider中间件处理过了，你处理一下（<strong>注意！这儿responses默认是交给def parse这个函数处理的</strong>） <strong>Spiders：（处理完毕数据之后对于需要跟进的URL）</strong>，Hi！<strong>引擎</strong>，这是我需要跟进的URL，将它的responses交给函数 def xxxx(self, responses)处理。还有这是我获取到的Item。 <strong>引擎</strong>：Hi ！<strong>Item Pipeline</strong> 我这儿有个item你帮我处理一下！<strong>调度器！</strong>这是我需要的URL你帮我处理下。然后从第四步开始循环，直到获取到你需要的信息， 注意！只有当调度器中不存在任何request了，整个程序才会停止，（也就是说，对于下载失败的ＵＲＬ，Scrapy会重新下载。） 以上就是Scrapy整个流程了。 <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/10/QQ图片20161022193315.gif" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/10/QQ图片20161022193315.gif" alt="QQ图片20161022193315"></a> 大家将就着看看。 建立一个项目之后： <strong>第一件事情</strong>是在items.py文件中定义一些字段，这些字段用来临时存储你需要保存的数据。方便后面保存数据到其他地方，比如数据库 或者 本地文本之类的。 <strong>第二件事情</strong>在spiders文件夹中编写自己的爬虫 <strong>第三件事情</strong>在pipelines.py中存储自己的数据 <strong>还有一件事情</strong>，不是非做不可的，就settings.py文件 并不是一定要编辑的，只有有需要的时候才会编辑。 <strong>建议一点：在大家调试的时候建议大家在settings.py中取消下面几行的注释：</strong> <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/12/设置setting01.png" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/12/设置setting01.png" alt="设置setting01"></a> 这几行注释的作用是，Scrapy会缓存你有的Requests!当你再次请求时，如果存在缓存文档则返回缓存文档，而不是去网站请求，这样既加快了本地调试速度，也减轻了 网站的压力。一举多得 <strong>第一步定义字段：</strong> 好了，我们来做 第一步 定义一些字段；那具体我们要定义那些字段呢？ 这个根据自己需要的提取的内容来定义。 比如：我们爬取小说站点都需要提取些什么数据啊？ 小说名字、作者、小说地址、连载状态、连载字数、文章类别 就像下面这样： <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/12/Scrapy01.png" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/12/Scrapy01.png" alt="Scrapy01"></a> 这样我们第一步就完成啦！是不是So Easy？ヾ(<em>´▽‘</em>)ﾉ ； 下面开始重点了哦！编写spider（就是我们用来提取数据的爬虫了） <strong>第二步编写Spider：</strong> 在spiders文件中新建一个dingdian.py文件 并导入我们需用的模块 <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/12/Scrapy02.png" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/12/Scrapy02.png" alt="Scrapy02"></a> <strong>PS：Scrapy中Response可以直接使用Xpath来解析数据；不过大家也可以使用自己习惯的包，比如我导入的BS4 、re ；当然也可以使其他比如pyquery之类的。这个并没有什么限制</strong> <strong>另外或许个别小伙伴会遇到 from dingdian.items import DingdianItem这个导入失败的情况；可以试试把项目文件移动到根目录。</strong> <strong>Request这个模块可以用来重写单独请求一个URL，用于我们后面跟进URL。</strong> 好了开整；首先我们需要什么？ 我们需要从一个地址入手开始爬取，我在顶点小说上没有发现有全站小说地址，但是我找到每个分类地址全部小说： 玄幻魔幻：<a href="http://www.23wx.com/class/1_1.html" target="_blank" rel="noopener">http://www.23wx.com/class/1_1.html</a> 武侠修真：<a href="http://www.23wx.com/class/2_1.html" target="_blank" rel="noopener">http://www.23wx.com/class/2_1.html</a> 都市言情：<a href="http://www.23wx.com/class/3_1.html" target="_blank" rel="noopener">http://www.23wx.com/class/3_1.html</a> 历史军事：<a href="http://www.23wx.com/class/4_1.html" target="_blank" rel="noopener">http://www.23wx.com/class/4_1.html</a> 侦探推理：<a href="http://www.23wx.com/class/5_1.html" target="_blank" rel="noopener">http://www.23wx.com/class/5_1.html</a> 网游动漫：<a href="http://www.23wx.com/class/6_1.html" target="_blank" rel="noopener">http://www.23wx.com/class/6_1.html</a> 科幻小说：<a href="http://www.23wx.com/class/7_1.html" target="_blank" rel="noopener">http://www.23wx.com/class/7_1.html</a> 恐怖灵异：<a href="http://www.23wx.com/class/8_1.html" target="_blank" rel="noopener">http://www.23wx.com/class/8_1.html</a> 散文诗词：<a href="http://www.23wx.com/class/9_1.html" target="_blank" rel="noopener">http://www.23wx.com/class/9_1.html</a> 其他：<a href="http://www.23wx.com/class/10_1.html" target="_blank" rel="noopener">http://www.23wx.com/class/10_1.html</a> 全本：<a href="http://www.23wx.com/quanben/1" target="_blank" rel="noopener">http://www.23wx.com/quanben/1</a> 好啦！入口地址我们找到了，现在开始写第一部分代码： 当然对于上面的地址，我们是可以直接全使用Start_urls这种列表全部请求，不过并不太美观，我需要把其中，有规律的部分，单独其他方式实现，比如字典之类的： <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/12/Scrapy22.png" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/12/Scrapy22.png" alt="Scrapy22"></a> 第十行：首先我们创建一个类 Myspider；这个类继承自scrapy.Spider（当然还有一些其他父类，继承各个父类后能实现的功能不一样）； 第十二行：我们定义name：dingdian （请注意，这name就是我们在entrypoint.py文件中的第三个参数！）！！！！<strong>请务必注意：此Name的！名字！在整个项目中有且只能有一个、名字不可重复！！！！</strong> 第十一行：我们定义了一个allowed_domains；这个不是必须的；但是在某写情况下需要用得到，比如使用爬取规则的时候就需要了；它的作用是只会跟进存在于allowed_domains中的URL。不存在的URL会被忽略。 第十七行到第十九行：我们使用字符串拼接的方式实现了我们上面发现的全部URL。 第二十行和二十一行：我们使用了导入的Request包，来跟进我们的URL（并将返回的response作为参数传递给self.parse, 嗯！这个叫<strong>回调函数</strong>！） 第二十三行：使用parse函数接受上面request获取到的response。（请务必注意：不要轻易改写parse函数（意思就是不要把parse函数用作它用）；因为这样request的回调函数被你用了，就没谁接受request返回的response啦！如果你非要用作它用，则需要自己给request一个回调函数哦！）</p>
                <figure class="highlight python">
                  <table>
                    <tr>
                      <td class="gutter">
                        <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br></pre>
                      </td>
                      <td class="code">
                        <pre><span class="line"><span class="keyword">import</span> re</span><br><span class="line"><span class="keyword">import</span> scrapy <span class="comment">#导入scrapy包</span></span><br><span class="line"><span class="keyword">from</span> bs4 <span class="keyword">import</span> BeautifulSoup</span><br><span class="line"><span class="keyword">from</span> scrapy.http <span class="keyword">import</span> Request <span class="comment">##一个单独的request的模块，需要跟进URL的时候，需要用它</span></span><br><span class="line"><span class="keyword">from</span> dingdian.items <span class="keyword">import</span> DingdianItem <span class="comment">##这是我定义的需要保存的字段，（导入dingdian项目中，items文件中的DingdianItem类）</span></span><br><span class="line"></span><br><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">Myspider</span><span class="params">(scrapy.Spider)</span>:</span></span><br><span class="line"></span><br><span class="line">    name = <span class="string">'dingdian'</span></span><br><span class="line">    allowed_domains = [<span class="string">'23wx.com'</span>]</span><br><span class="line">    bash_url = <span class="string">'http://www.23wx.com/class/'</span></span><br><span class="line">    bashurl = <span class="string">'.html'</span></span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">start_requests</span><span class="params">(self)</span>:</span></span><br><span class="line">        <span class="keyword">for</span> i <span class="keyword">in</span> range(<span class="number">1</span>, <span class="number">11</span>):</span><br><span class="line">            url = self.bash_url + str(i) + <span class="string">'_1'</span> + self.bashurl</span><br><span class="line">            <span class="keyword">yield</span> Request(url, self.parse)</span><br><span class="line">        <span class="keyword">yield</span> Request(<span class="string">'http://www.23wx.com/quanben/1'</span>, self.parse)</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">parse</span><span class="params">(self, response)</span>:</span></span><br><span class="line">        print(response.text)</span><br></pre>
                      </td>
                    </tr>
                  </table>
                </figure>
                <p>我们测试一下是否正常工作：在IDE中运行我们之前创建的entrypoint.py文件（如果没有这个文件是不能在IDE中运行的哦！ヽ(=^･ω･^=)丿） 然后会像这样： <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/11/Spider编写03.png" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/11/Spider编写03.png" alt="Spider编写03"></a> 你会发现在红色状态报告之后，所有页面几乎是一瞬间出现的；那是因为Scrapy使用了异步啦！ヽ(°◇° )ノ 另外因为Scrapy遵循了robots规则，如果你想要获取的页面在robots中被禁止了，Scrapy是会忽略掉的哦！！ヾ(。￣□￣)ﾂ゜゜゜ 请求就这么轻而易举的实现了啊！简直So Easy！ 继续 继续！ 我们需要历遍所有页面才能取得所有的小说页面连接： <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/11/分析网页2.png" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/11/分析网页2.png" alt="分析网页2"></a> <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/11/分析网页01.png" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/11/分析网页01.png" alt="分析网页01"></a> 每个页面的这个位置都是最后一个页面，我们提取出它，历遍就可以拼接出一个完整的URL了ヾ§ ￣▽)ゞ2333333 Go Go <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/12/Scrapy20.png" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/12/Scrapy20.png" alt="Scrapy20"></a> 第二十三行：def parse(self, response)这个函数接受来在二十一行返回的response，并处理。 第二十四行：我们使用BS4从response中获取到了最大页码。 第二十五行至二十七行：我们照例拼接了一个完整的URL（response.url:就是这个response的URL地址） 第二十八行：功能和第二十行一样，callback= 是指定回调函数，不过不写callback=也没有什么影响！ 注意我只是说的callback=这个几个；不是后面的self.get_name. 看清楚了response是怎么用的没？ヾ§ ￣▽)ゞ2333333是不是So Easy？ 如果不清楚那个拼接URL的小伙伴可以打印出来，看看哦··· 再去观察一下网页，就很明白啦 上面两个函数就彻底的把整个网站的所有小说的页面URL的提取出来了，并将每个页面的response交给了get_name函数处理哦！ 现在我们的爬虫就开始处理具体的小说了哦： <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/12/Scrapy07.png" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/12/Scrapy07.png" alt="Scrapy07"></a> 瞅见没 我们需要的东西，快用F12工具看一下吧，在什么位置有什么标签，可以方便我们提取数据。还不知道怎么看的小伙伴，去看看妹子图那个教程，有教哦；实在不行百度一下也行！ 过程忽略了，直接上代码（主要是懒癌来了）： <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/12/Scrapy09.png" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/12/Scrapy09.png" alt="Scrapy09"></a> 前面三行不说了， 第三十七和三十八行：是我们的小说名字和URL 第三十九行和第四十行；大伙儿可能会发现，多了个一个meta这么一个字典，这是Scrapy中传递额外数据的方法。因我们还有一些其他内容需要在下一个页面中才能获取到。 下面我的爬虫进入了这个页面： <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/12/Scrapy10.png" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/12/Scrapy10.png" alt="Scrapy10"></a> 这个页面就有很多我们需要的信息了：废话不说了代码上来： <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/12/Scrapy11.png" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/12/Scrapy11.png" alt="Scrapy11"></a> 第四十行：将我们导入的item文件进行实例化，用来存储我们的数据。 后面全部：将需要的数据，复制给item[key] (注意这儿的Key就是我们前面在item文件中定义的那些字段。) 注意！response.meta[key]：这个是提取从上一个函数传递下来的值。 return item 就是返回我们的字典了，然后Pipelines就可以开始对这些数据进行处理了。比如 存储之类的。 好啦，Spiders我们先编写到这个地方。（是不是有小伙伴发现我还有几个字段没有取值？当然留着你们自己试试了，哈哈哈ヽ(=^･ω･^=)丿）后面再继续。 我现在教教大家怎么处理这些数据：对头就是说说Pipeline了： 对于基本的Pipeline存储方式，网上有很多教程了，今天我们做一个自定义的MySQL的Pipeline： 首先为了能好区分框架自带的Pipeline，我们把MySQL的Pipeline单独放到一个目录里面。 <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/12/Scrapy12.png" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/12/Scrapy12.png" alt="Scrapy12"></a> 我们在项目中新建了一个mysqlpipelines的文件夹，我们所有的MySQL文件都放在这个目录。 <strong>init</strong>.py 这个文件不需要我说了吧，不知道做啥的小哥儿自己百度。 pipelines.py 这个是我们写存放数据的文件 sql.py 看名字就知道，需要的sql语句。 首先是需要的MySQL表，（MySQL都没有的小哥儿 自己百度装一个啊，我就不教了）</p>
                <figure class="highlight routeros">
                  <table>
                    <tr>
                      <td class="gutter">
                        <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre>
                      </td>
                      <td class="code">
                        <pre><span class="line">DROP TABLE <span class="keyword">IF</span> EXISTS `dd_name`;</span><br><span class="line">CREATE TABLE `dd_name` (</span><br><span class="line">  `id` int(11) <span class="keyword">NOT</span> <span class="literal">NULL</span> AUTO_INCREMENT,</span><br><span class="line">  `xs_name` varchar(255)<span class="built_in"> DEFAULT </span><span class="literal">NULL</span>,</span><br><span class="line">  `xs_author` varchar(255)<span class="built_in"> DEFAULT </span><span class="literal">NULL</span>,</span><br><span class="line">  `category` varchar(255)<span class="built_in"> DEFAULT </span><span class="literal">NULL</span>,</span><br><span class="line">  `name_id` varchar(255)<span class="built_in"> DEFAULT </span><span class="literal">NULL</span>,</span><br><span class="line">  PRIMARY KEY (`id`)</span><br><span class="line">) <span class="attribute">ENGINE</span>=InnoDB <span class="attribute">AUTO_INCREMENT</span>=38<span class="built_in"> DEFAULT </span><span class="attribute">CHARSET</span>=utf8mb4;</span><br></pre>
                      </td>
                    </tr>
                  </table>
                </figure>
                <p>首先我们再settings.py文件中定义好MySQL的配置文件（当然你也可以直接定义在sql.py文件中）： <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/12/MySQL-setting.png" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/12/MySQL-setting.png" alt="MySQL setting"></a> <strong>PS :注意MySQL的默认端口是 3306；我自己的MySQL改成了3389。这儿各位酌情自己更改。</strong> 在开始写sql.py之前，我们需要安装一个Python操作MySQL的包，来自MySQL官方的一个包：<a href="http://cdn.mysql.com//Downloads/Connector-Python/mysql-connector-python-2.1.4.zip" target="_blank" rel="noopener">点我下载</a> 下载完成后解压出来，从CMD进入该目录的绝对路径，然后 Python setup.py install ；即可完成安装 下面是我们的sql.py文件： <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/12/sql01.png" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/12/sql01.png" alt="sql01"></a> 第一行至第二行分别导入了：我的MySQL操作包，settings配置文件 第四行至第八行 ： 从settings配置文件中获取到了，我们的MySQL配置文件 第十行至第十一行： 初始化了一个MySQL的操作游标 第十三行： 定义了一个Sql的类 第十六行至第二十五行：定义了一个函数，将函数中的四个变量写入数据库（这四个变量就是我们后面传进来的需要存储的数据。） <strong>关于@classmethod这个是一个修饰符；作用是我们不需要初始化类就可以直接调用类中的函数使用（具体说起来麻烦，知道作用就好啦）</strong> 好了第一部分写完了，我们还需要一个能够去重的： <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/12/sql01-1.png" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/12/sql01-1.png" alt="sql01"></a> 这一段代码会查找name_id这个字段，如果存在则会返回 1 不存在则会返回0 Nice！sqi.py这一部分我们完成，来开始写pipeline吧： <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/12/pipeline02.png" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/12/pipeline02.png" alt="pipeline02"></a> 第一行至第二行：我们导入了之前编写的sql.py中的Sql类，和我们建立的item 第六行：建立了一个DingdianPipeline的类（别忘了一定要继承object这个类啊，这是做啥的不用多了解，说多了你们头晕，我也懒） 第八行：我们定义了一个process_item函数并有，item和spider这两个参数（请注意啊！这两玩意儿 务必！！！要加上！！千万不能少！！！！务必！！！务必！！！） 第十行：你这样理解如果在 item中存在DingdianItem；就执行下面的。 第十一行：从item中取出 name_id的值。 第十二行：调用Sql中的select_name函数获得返回值 第十三行：判断ret是否等于1 ，是的话证明已经存了 第二十行：调用Sql中的 insert_dd_name函数，存储上面几个值。 搞完！下面我们启用这个Pipeline在settings中作如下设置： <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/12/setting02.png" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/12/setting02.png" alt="setting02"></a> <strong>PS: dingdian（项目目录）.mysqlpipelines（自己建立的MySQL目录）.pipelines（自己建立的pipelines文件）.DingdianPipeline（其中定义的类） 后面的 1 是优先级程度（1-1000随意设置，数值越低，组件的优先级越高）</strong> 好！我们来运行一下试试！！Go Go Go！ <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/12/scrapy15.png" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/12/scrapy15.png" alt="scrapy15"></a> Nice!!完美！！我之前运行过了 所以提示已经存在。 <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/12/scrapy17.png" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/12/scrapy17.png" alt="scrapy17"></a> 下面我们开始还剩下的一些内容获取：小说章节 和章节内容 首先我们在item中新定义一些需要获取内容的字段： <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/12/scrapy16.png" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/12/scrapy16.png" alt="scrapy16"></a> 代码不解释了哦！（懒癌来了，写不下去了） 继续编写Spider文件： <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/12/scrapy18.png" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/12/scrapy18.png" alt="scrapy18"></a> 请注意我图中画红框的的地方，这个地方返回item是不能用return的哦！用了就结束了，程序就不会继续下去了，得用yield（你知道就行，这玩意儿说起来麻烦。） 第五十八行： num这个变量的作用是 因为Scrapy是异步的方式运作，你采集到的章节顺序都是混乱的，需要给它有序的序列，我们按照这个排序就能得到正确的章节顺序啦 请注意在顶部导入定义的第二个item类！ 下面我们来写存储这部分spider的Pipeline： 数据表：</p>
                <figure class="highlight routeros">
                  <table>
                    <tr>
                      <td class="gutter">
                        <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br></pre>
                      </td>
                      <td class="code">
                        <pre><span class="line">DROP TABLE <span class="keyword">IF</span> EXISTS `dd_chaptername`;</span><br><span class="line">CREATE TABLE `dd_chaptername` (</span><br><span class="line">  `id` int(11) <span class="keyword">NOT</span> <span class="literal">NULL</span> AUTO_INCREMENT,</span><br><span class="line">  `xs_chaptername` varchar(255)<span class="built_in"> DEFAULT </span><span class="literal">NULL</span>,</span><br><span class="line">  `xs_content` text,</span><br><span class="line">  `id_name` int(11)<span class="built_in"> DEFAULT </span><span class="literal">NULL</span>,</span><br><span class="line">  `num_id` int(11)<span class="built_in"> DEFAULT </span><span class="literal">NULL</span>,</span><br><span class="line">  `url` varchar(255)<span class="built_in"> DEFAULT </span><span class="literal">NULL</span>,</span><br><span class="line">  PRIMARY KEY (`id`)</span><br><span class="line">) <span class="attribute">ENGINE</span>=InnoDB <span class="attribute">AUTO_INCREMENT</span>=2726<span class="built_in"> DEFAULT </span><span class="attribute">CHARSET</span>=gb18030;</span><br><span class="line"><span class="builtin-name">SET</span> <span class="attribute">FOREIGN_KEY_CHECKS</span>=1;</span><br></pre>
                      </td>
                    </tr>
                  </table>
                </figure>
                <p> Sql.py: <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/12/Scrapy13.png" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/12/Scrapy13.png" alt="Scrapy13"></a> <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/12/Scrapy14.png" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/12/Scrapy14.png" alt="Scrapy14"></a> 不解释了哦！ 下面是Pipeline： <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/12/scrapy21.png" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/12/scrapy21.png" alt="scrapy21"></a> 有小伙伴注意，这儿比上面一个Pipeline少一个判断，因为我把判断移动到Spider中去了，这样就可以减少一次Request，减轻服务器压力。 改变后的Spider长这样： <a href="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/12/Scrapy16.png" target="_blank" rel="noopener"><img src="http://qiniu.cuiqingcai.com/wp-content/uploads/2016/12/Scrapy16.png" alt="Scrapy16"></a> 别忘了在spider中导入Sql哦！ヾ(。￣□￣)ﾂ゜゜゜ 到此收工！！！！ 至于小说图片，因为Scrapy的图片下载管道，是自动以md5命名，而且感觉不爽··· 后面单独写一个异步下载的脚本··· <a href="https://github.com/thsheep/dingdian" target="_blank" rel="noopener">https://github.com/thsheep/dingdian</a></p>
              </div>
              <div class="reward-container">
                <div></div>
                <button onclick="var qr = document.getElementById('qr'); qr.style.display = (qr.style.display === 'none') ? 'block' : 'none';"> 打赏 </button>
                <div id="qr" style="display: none;">
                  <div style="display: inline-block;">
                    <img src="/images/wechatpay.jpg" alt="崔庆才 微信支付">
                    <p>微信支付</p>
                  </div>
                  <div style="display: inline-block;">
                    <img src="/images/alipay.jpg" alt="崔庆才 支付宝">
                    <p>支付宝</p>
                  </div>
                </div>
              </div>
              <footer class="post-footer">
                <div class="post-nav">
                  <div class="post-nav-item">
                    <a href="/3494.html" rel="prev" title="Composer进阶使用之常用命令和版本约束">
                      <i class="fa fa-chevron-left"></i> Composer进阶使用之常用命令和版本约束 </a>
                  </div>
                  <div class="post-nav-item">
                    <a href="/3801.html" rel="next" title="使用Python收集获取Linux系统主机信息"> 使用Python收集获取Linux系统主机信息 <i class="fa fa-chevron-right"></i>
                    </a>
                  </div>
                </div>
              </footer>
            </article>
          </div>
          <div class="comments" id="gitalk-container"></div>
          <script>
            window.addEventListener('tabs:register', () =>
            {
              let
              {
                activeClass
              } = CONFIG.comments;
              if (CONFIG.comments.storage)
              {
                activeClass = localStorage.getItem('comments_active') || activeClass;
              }
              if (activeClass)
              {
                let activeTab = document.querySelector(`a[href="#comment-${activeClass}"]`);
                if (activeTab)
                {
                  activeTab.click();
                }
              }
            });
            if (CONFIG.comments.storage)
            {
              window.addEventListener('tabs:click', event =>
              {
                if (!event.target.matches('.tabs-comment .tab-content .tab-pane')) return;
                let commentClass = event.target.classList[1];
                localStorage.setItem('comments_active', commentClass);
              });
            }

          </script>
        </div>
        <div class="toggle sidebar-toggle">
          <span class="toggle-line toggle-line-first"></span>
          <span class="toggle-line toggle-line-middle"></span>
          <span class="toggle-line toggle-line-last"></span>
        </div>
        <aside class="sidebar">
          <div class="sidebar-inner">
            <ul class="sidebar-nav motion-element">
              <li class="sidebar-nav-toc"> 文章目录 </li>
              <li class="sidebar-nav-overview"> 站点概览 </li>
            </ul>
            <!--noindex-->
            <div class="post-toc-wrap sidebar-panel">
            </div>
            <!--/noindex-->
            <div class="site-overview-wrap sidebar-panel">
              <div class="site-author motion-element" itemprop="author" itemscope itemtype="http://schema.org/Person">
                <img class="site-author-image" itemprop="image" alt="崔庆才" src="/images/avatar.png">
                <p class="site-author-name" itemprop="name">崔庆才</p>
                <div class="site-description" itemprop="description">崔庆才的个人站点，记录生活的瞬间，分享学习的心得。</div>
              </div>
              <div class="site-state-wrap motion-element">
                <nav class="site-state">
                  <div class="site-state-item site-state-posts">
                    <a href="/archives/">
                      <span class="site-state-item-count">608</span>
                      <span class="site-state-item-name">日志</span>
                    </a>
                  </div>
                  <div class="site-state-item site-state-categories">
                    <a href="/categories/">
                      <span class="site-state-item-count">24</span>
                      <span class="site-state-item-name">分类</span></a>
                  </div>
                  <div class="site-state-item site-state-tags">
                    <a href="/tags/">
                      <span class="site-state-item-count">156</span>
                      <span class="site-state-item-name">标签</span></a>
                  </div>
                </nav>
              </div>
              <div class="links-of-author motion-element">
                <span class="links-of-author-item">
                  <a href="https://github.com/Germey" title="GitHub → https:&#x2F;&#x2F;github.com&#x2F;Germey" rel="noopener" target="_blank"><i class="fab fa-github fa-fw"></i>GitHub</a>
                </span>
                <span class="links-of-author-item">
                  <a href="mailto:cqc@cuiqingcai.com.com" title="邮件 → mailto:cqc@cuiqingcai.com.com" rel="noopener" target="_blank"><i class="fa fa-envelope fa-fw"></i>邮件</a>
                </span>
                <span class="links-of-author-item">
                  <a href="https://weibo.com/cuiqingcai" title="微博 → https:&#x2F;&#x2F;weibo.com&#x2F;cuiqingcai" rel="noopener" target="_blank"><i class="fab fa-weibo fa-fw"></i>微博</a>
                </span>
                <span class="links-of-author-item">
                  <a href="https://www.zhihu.com/people/Germey" title="知乎 → https:&#x2F;&#x2F;www.zhihu.com&#x2F;people&#x2F;Germey" rel="noopener" target="_blank"><i class="fa fa-magic fa-fw"></i>知乎</a>
                </span>
              </div>
            </div>
            <div style=" width: 100%;" class="sidebar-panel sidebar-panel-image sidebar-panel-active">
              <a href="https://tutorial.lengyue.video/?coupon=12ef4b1a-a3db-11ea-bb37-0242ac130002_cqx_850" target="_blank" rel="noopener">
                <img src="https://qiniu.cuiqingcai.com/bco2a.png" style=" width: 100%;">
              </a>
            </div>
            <div style=" width: 100%;" class="sidebar-panel sidebar-panel-image sidebar-panel-active">
              <a href="http://www.ipidea.net/?utm-source=cqc&utm-keyword=?cqc" target="_blank" rel="noopener">
                <img src="https://qiniu.cuiqingcai.com/0ywun.png" style=" width: 100%;">
              </a>
            </div>
            <div class="sidebar-panel sidebar-panel-tags sidebar-panel-active">
              <h4 class="name"> 标签云 </h4>
              <div class="content">
                <a href="/tags/2048/" style="font-size: 10px;">2048</a> <a href="/tags/API/" style="font-size: 10px;">API</a> <a href="/tags/Bootstrap/" style="font-size: 11.25px;">Bootstrap</a> <a href="/tags/CDN/" style="font-size: 10px;">CDN</a> <a href="/tags/CQC/" style="font-size: 10px;">CQC</a> <a href="/tags/CSS/" style="font-size: 10px;">CSS</a> <a href="/tags/CSS-%E5%8F%8D%E7%88%AC%E8%99%AB/" style="font-size: 10px;">CSS 反爬虫</a> <a href="/tags/CV/" style="font-size: 10px;">CV</a> <a href="/tags/Django/" style="font-size: 10px;">Django</a> <a href="/tags/Eclipse/" style="font-size: 11.25px;">Eclipse</a> <a href="/tags/FTP/" style="font-size: 10px;">FTP</a> <a href="/tags/Git/" style="font-size: 10px;">Git</a> <a href="/tags/GitHub/" style="font-size: 13.75px;">GitHub</a> <a href="/tags/HTML5/" style="font-size: 10px;">HTML5</a> <a href="/tags/Hexo/" style="font-size: 10px;">Hexo</a> <a href="/tags/IT/" style="font-size: 10px;">IT</a> <a href="/tags/JSP/" style="font-size: 10px;">JSP</a> <a href="/tags/JavaScript/" style="font-size: 10px;">JavaScript</a> <a href="/tags/K8s/" style="font-size: 10px;">K8s</a> <a href="/tags/LOGO/" style="font-size: 10px;">LOGO</a> <a href="/tags/Linux/" style="font-size: 10px;">Linux</a> <a href="/tags/MIUI/" style="font-size: 10px;">MIUI</a> <a href="/tags/MongoDB/" style="font-size: 10px;">MongoDB</a> <a href="/tags/Mysql/" style="font-size: 10px;">Mysql</a> <a href="/tags/NBA/" style="font-size: 10px;">NBA</a> <a href="/tags/PHP/" style="font-size: 11.25px;">PHP</a> <a href="/tags/PS/" style="font-size: 10px;">PS</a> <a href="/tags/Pathlib/" style="font-size: 10px;">Pathlib</a> <a href="/tags/PhantomJS/" style="font-size: 10px;">PhantomJS</a> <a href="/tags/Python/" style="font-size: 15px;">Python</a> <a href="/tags/Python3/" style="font-size: 12.5px;">Python3</a> <a href="/tags/Pythonic/" style="font-size: 10px;">Pythonic</a> <a href="/tags/QQ/" style="font-size: 10px;">QQ</a> <a href="/tags/Redis/" style="font-size: 10px;">Redis</a> <a href="/tags/SAE/" style="font-size: 10px;">SAE</a> <a href="/tags/SSH/" style="font-size: 10px;">SSH</a> <a href="/tags/SVG/" style="font-size: 10px;">SVG</a> <a href="/tags/Scrapy/" style="font-size: 10px;">Scrapy</a> <a href="/tags/Scrapy-redis/" style="font-size: 10px;">Scrapy-redis</a> <a href="/tags/Scrapy%E5%88%86%E5%B8%83%E5%BC%8F/" style="font-size: 10px;">Scrapy分布式</a> <a href="/tags/Selenium/" style="font-size: 10px;">Selenium</a> <a href="/tags/TKE/" style="font-size: 10px;">TKE</a> <a href="/tags/Ubuntu/" style="font-size: 11.25px;">Ubuntu</a> <a href="/tags/VS-Code/" style="font-size: 10px;">VS Code</a> <a href="/tags/Vs-Code/" style="font-size: 10px;">Vs Code</a> <a href="/tags/Vue/" style="font-size: 11.25px;">Vue</a> <a href="/tags/Webpack/" style="font-size: 10px;">Webpack</a> <a href="/tags/Windows/" style="font-size: 10px;">Windows</a> <a href="/tags/Winpcap/" style="font-size: 10px;">Winpcap</a> <a href="/tags/WordPress/" style="font-size: 13.75px;">WordPress</a> <a href="/tags/Youtube/" style="font-size: 11.25px;">Youtube</a> <a href="/tags/android/" style="font-size: 10px;">android</a> <a href="/tags/ansible/" style="font-size: 10px;">ansible</a> <a href="/tags/cocos2d-x/" style="font-size: 10px;">cocos2d-x</a> <a href="/tags/e6/" style="font-size: 10px;">e6</a> <a href="/tags/fitvids/" style="font-size: 10px;">fitvids</a> <a href="/tags/git/" style="font-size: 11.25px;">git</a> <a href="/tags/json/" style="font-size: 10px;">json</a> <a href="/tags/js%E9%80%86%E5%90%91/" style="font-size: 10px;">js逆向</a> <a href="/tags/kubernetes/" style="font-size: 10px;">kubernetes</a> <a href="/tags/log/" style="font-size: 10px;">log</a> <a href="/tags/logging/" style="font-size: 10px;">logging</a> <a href="/tags/matlab/" style="font-size: 11.25px;">matlab</a> <a href="/tags/python/" style="font-size: 20px;">python</a> <a href="/tags/pytube/" style="font-size: 11.25px;">pytube</a> <a href="/tags/pywin32/" style="font-size: 10px;">pywin32</a> <a href="/tags/style/" style="font-size: 10px;">style</a> <a href="/tags/tomcat/" style="font-size: 10px;">tomcat</a> <a href="/tags/ubuntu/" style="font-size: 10px;">ubuntu</a> <a href="/tags/uwsgi/" style="font-size: 10px;">uwsgi</a> <a href="/tags/vsftpd/" style="font-size: 10px;">vsftpd</a> <a href="/tags/wamp/" style="font-size: 10px;">wamp</a> <a href="/tags/wineQQ/" style="font-size: 10px;">wineQQ</a> <a href="/tags/%E4%B8%83%E7%89%9B/" style="font-size: 11.25px;">七牛</a> <a href="/tags/%E4%B8%8A%E6%B5%B7/" style="font-size: 10px;">上海</a> <a href="/tags/%E4%B8%AA%E4%BA%BA%E7%BD%91%E7%AB%99/" style="font-size: 10px;">个人网站</a> <a href="/tags/%E4%B8%BB%E9%A2%98/" style="font-size: 10px;">主题</a> <a href="/tags/%E4%BA%91%E4%BA%A7%E5%93%81/" style="font-size: 10px;">云产品</a> <a href="/tags/%E4%BA%91%E5%AD%98%E5%82%A8/" style="font-size: 10px;">云存储</a> <a href="/tags/%E4%BA%AC%E4%B8%9C%E4%BA%91/" style="font-size: 10px;">京东云</a> <a href="/tags/%E4%BA%BA%E5%B7%A5%E6%99%BA%E8%83%BD/" style="font-size: 12.5px;">人工智能</a> <a href="/tags/%E4%BB%A3%E7%90%86/" style="font-size: 10px;">代理</a> <a href="/tags/%E4%BB%A3%E7%A0%81/" style="font-size: 10px;">代码</a> <a href="/tags/%E4%BB%A3%E7%A0%81%E5%88%86%E4%BA%AB%E5%9B%BE/" style="font-size: 10px;">代码分享图</a> <a href="/tags/%E4%BC%98%E5%8C%96/" style="font-size: 10px;">优化</a> <a href="/tags/%E4%BD%8D%E8%BF%90%E7%AE%97/" style="font-size: 10px;">位运算</a> <a href="/tags/%E5%85%AC%E4%BC%97%E5%8F%B7/" style="font-size: 10px;">公众号</a> <a href="/tags/%E5%88%86%E4%BA%AB/" style="font-size: 10px;">分享</a> <a href="/tags/%E5%88%86%E5%B8%83%E5%BC%8F/" style="font-size: 10px;">分布式</a> <a href="/tags/%E5%88%9B%E4%B8%9A/" style="font-size: 10px;">创业</a> <a href="/tags/%E5%89%8D%E7%AB%AF/" style="font-size: 12.5px;">前端</a> <a href="/tags/%E5%8D%9A%E5%AE%A2/" style="font-size: 10px;">博客</a> <a href="/tags/%E5%8E%9F%E7%94%9FAPP/" style="font-size: 10px;">原生APP</a> <a href="/tags/%E5%8F%8D%E7%88%AC%E8%99%AB/" style="font-size: 12.5px;">反爬虫</a> <a href="/tags/%E5%91%BD%E4%BB%A4/" style="font-size: 10px;">命令</a> <a href="/tags/%E5%93%8D%E5%BA%94%E5%BC%8F%E5%B8%83%E5%B1%80/" style="font-size: 10px;">响应式布局</a> <a href="/tags/%E5%9E%83%E5%9C%BE%E9%82%AE%E4%BB%B6/" style="font-size: 10px;">垃圾邮件</a> <a href="/tags/%E5%9F%9F%E5%90%8D%E7%BB%91%E5%AE%9A/" style="font-size: 10px;">域名绑定</a> <a href="/tags/%E5%A4%8D%E7%9B%98/" style="font-size: 10px;">复盘</a> <a href="/tags/%E5%A4%A7%E4%BC%97%E7%82%B9%E8%AF%84/" style="font-size: 10px;">大众点评</a> <a href="/tags/%E5%AD%97%E4%BD%93%E5%8F%8D%E7%88%AC%E8%99%AB/" style="font-size: 10px;">字体反爬虫</a> <a href="/tags/%E5%AD%97%E7%AC%A6%E9%97%AE%E9%A2%98/" style="font-size: 10px;">字符问题</a> <a href="/tags/%E5%AD%A6%E4%B9%A0%E6%96%B9%E6%B3%95/" style="font-size: 10px;">学习方法</a> <a href="/tags/%E5%AE%89%E5%8D%93/" style="font-size: 10px;">安卓</a> <a href="/tags/%E5%AE%9E%E7%94%A8/" style="font-size: 10px;">实用</a> <a href="/tags/%E5%B0%81%E9%9D%A2/" style="font-size: 10px;">封面</a> <a href="/tags/%E5%B4%94%E5%BA%86%E6%89%8D/" style="font-size: 18.75px;">崔庆才</a> <a href="/tags/%E5%B7%A5%E5%85%B7/" style="font-size: 12.5px;">工具</a> <a href="/tags/%E5%BC%80%E5%8F%91%E5%B7%A5%E5%85%B7/" style="font-size: 10px;">开发工具</a> <a href="/tags/%E5%BE%AE%E8%BD%AF/" style="font-size: 10px;">微软</a> <a href="/tags/%E6%80%9D%E8%80%83/" style="font-size: 10px;">思考</a> <a href="/tags/%E6%89%8B%E6%9C%BA%E8%AE%BF%E9%97%AE/" style="font-size: 10px;">手机访问</a> <a href="/tags/%E6%95%99%E7%A8%8B/" style="font-size: 10px;">教程</a> <a href="/tags/%E6%95%99%E8%82%B2/" style="font-size: 10px;">教育</a> <a href="/tags/%E6%96%B0%E4%B9%A6/" style="font-size: 12.5px;">新书</a> <a href="/tags/%E6%96%B9%E6%B3%95%E8%AE%BA/" style="font-size: 10px;">方法论</a> <a href="/tags/%E6%97%85%E6%B8%B8/" style="font-size: 10px;">旅游</a> <a href="/tags/%E6%97%A5%E5%BF%97/" style="font-size: 10px;">日志</a> <a href="/tags/%E6%9A%97%E6%97%B6%E9%97%B4/" style="font-size: 10px;">暗时间</a> <a href="/tags/%E6%9D%9C%E5%85%B0%E7%89%B9/" style="font-size: 11.25px;">杜兰特</a> <a href="/tags/%E6%A1%8C%E9%9D%A2/" style="font-size: 10px;">桌面</a> <a href="/tags/%E6%AD%8C%E5%8D%95/" style="font-size: 10px;">歌单</a> <a href="/tags/%E6%B1%9F%E5%8D%97/" style="font-size: 10px;">江南</a> <a href="/tags/%E6%B8%B8%E6%88%8F/" style="font-size: 10px;">游戏</a> <a href="/tags/%E7%84%A6%E8%99%91/" style="font-size: 10px;">焦虑</a> <a href="/tags/%E7%88%AC%E8%99%AB/" style="font-size: 16.25px;">爬虫</a> <a href="/tags/%E7%88%AC%E8%99%AB%E4%B9%A6%E7%B1%8D/" style="font-size: 11.25px;">爬虫书籍</a> <a href="/tags/%E7%8E%AF%E5%A2%83%E5%8F%98%E9%87%8F/" style="font-size: 10px;">环境变量</a> <a href="/tags/%E7%94%9F%E6%B4%BB%E7%AC%94%E8%AE%B0/" style="font-size: 10px;">生活笔记</a> <a href="/tags/%E7%99%BB%E5%BD%95/" style="font-size: 10px;">登录</a> <a href="/tags/%E7%9F%A5%E4%B9%8E/" style="font-size: 10px;">知乎</a> <a href="/tags/%E7%9F%AD%E4%BF%A1/" style="font-size: 10px;">短信</a> <a href="/tags/%E7%9F%AD%E4%BF%A1%E9%AA%8C%E8%AF%81%E7%A0%81/" style="font-size: 10px;">短信验证码</a> <a href="/tags/%E7%AC%94%E8%AE%B0%E8%BD%AF%E4%BB%B6/" style="font-size: 10px;">笔记软件</a> <a href="/tags/%E7%AF%AE%E7%BD%91/" style="font-size: 10px;">篮网</a> <a href="/tags/%E7%BA%B8%E5%BC%A0/" style="font-size: 10px;">纸张</a> <a href="/tags/%E7%BB%84%E4%BB%B6/" style="font-size: 10px;">组件</a> <a href="/tags/%E7%BD%91%E7%AB%99/" style="font-size: 10px;">网站</a> <a href="/tags/%E7%BD%91%E7%BB%9C%E7%88%AC%E8%99%AB/" style="font-size: 11.25px;">网络爬虫</a> <a href="/tags/%E7%BE%8E%E5%AD%A6/" style="font-size: 10px;">美学</a> <a href="/tags/%E8%82%89%E5%A4%B9%E9%A6%8D/" style="font-size: 10px;">肉夹馍</a> <a href="/tags/%E8%85%BE%E8%AE%AF%E4%BA%91/" style="font-size: 10px;">腾讯云</a> <a href="/tags/%E8%87%AA%E5%BE%8B/" style="font-size: 10px;">自律</a> <a href="/tags/%E8%A5%BF%E5%B0%91%E7%88%B7/" style="font-size: 10px;">西少爷</a> <a href="/tags/%E8%A7%86%E9%A2%91/" style="font-size: 10px;">视频</a> <a href="/tags/%E8%B0%B7%E6%AD%8C%E9%AA%8C%E8%AF%81%E7%A0%81/" style="font-size: 10px;">谷歌验证码</a> <a href="/tags/%E8%BF%90%E8%90%A5/" style="font-size: 10px;">运营</a> <a href="/tags/%E8%BF%9C%E7%A8%8B/" style="font-size: 10px;">远程</a> <a href="/tags/%E9%80%86%E5%90%91/" style="font-size: 10px;">逆向</a> <a href="/tags/%E9%85%8D%E7%BD%AE/" style="font-size: 10px;">配置</a> <a href="/tags/%E9%87%8D%E8%A3%85/" style="font-size: 10px;">重装</a> <a href="/tags/%E9%98%BF%E6%9D%9C/" style="font-size: 10px;">阿杜</a> <a href="/tags/%E9%9D%99%E8%A7%85/" style="font-size: 17.5px;">静觅</a> <a href="/tags/%E9%A2%A0%E8%A6%86/" style="font-size: 10px;">颠覆</a> <a href="/tags/%E9%A3%9E%E4%BF%A1/" style="font-size: 10px;">飞信</a> <a href="/tags/%E9%B8%BF%E8%92%99/" style="font-size: 10px;">鸿蒙</a>
              </div>
              <script>
                const tagsColors = ['#00a67c', '#5cb85c', '#d9534f', '#567e95', '#b37333', '#f4843d', '#15a287']
                const tagsElements = document.querySelectorAll('.sidebar-panel-tags .content a')
                tagsElements.forEach((item) =>
                {
                  item.style.backgroundColor = tagsColors[Math.floor(Math.random() * tagsColors.length)]
                })

              </script>
            </div>
            <div class="sidebar-panel sidebar-panel-categories sidebar-panel-active">
              <h4 class="name"> 分类 </h4>
              <div class="content">
                <ul class="category-list">
                  <li class="category-list-item"><a class="category-list-link" href="/categories/C-C/">C/C++</a><span class="category-list-count">23</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/HTML/">HTML</a><span class="category-list-count">14</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/Java/">Java</a><span class="category-list-count">5</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/JavaScript/">JavaScript</a><span class="category-list-count">26</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/Linux/">Linux</a><span class="category-list-count">15</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/Markdown/">Markdown</a><span class="category-list-count">1</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/Net/">Net</a><span class="category-list-count">4</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/Other/">Other</a><span class="category-list-count">39</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/PHP/">PHP</a><span class="category-list-count">27</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/Paper/">Paper</a><span class="category-list-count">2</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/Python/">Python</a><span class="category-list-count">261</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/TypeScript/">TypeScript</a><span class="category-list-count">2</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/%E4%B8%AA%E4%BA%BA%E5%B1%95%E7%A4%BA/">个人展示</a><span class="category-list-count">1</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/%E4%B8%AA%E4%BA%BA%E6%97%A5%E8%AE%B0/">个人日记</a><span class="category-list-count">9</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/%E4%B8%AA%E4%BA%BA%E8%AE%B0%E5%BD%95/">个人记录</a><span class="category-list-count">4</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/%E4%B8%AA%E4%BA%BA%E9%9A%8F%E7%AC%94/">个人随笔</a><span class="category-list-count">15</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/%E5%AE%89%E8%A3%85%E9%85%8D%E7%BD%AE/">安装配置</a><span class="category-list-count">59</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/%E6%8A%80%E6%9C%AF%E6%9D%82%E8%B0%88/">技术杂谈</a><span class="category-list-count">88</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/%E6%9C%AA%E5%88%86%E7%B1%BB/">未分类</a><span class="category-list-count">1</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/%E7%94%9F%E6%B4%BB%E7%AC%94%E8%AE%B0/">生活笔记</a><span class="category-list-count">1</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/%E7%A6%8F%E5%88%A9%E4%B8%93%E5%8C%BA/">福利专区</a><span class="category-list-count">6</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/%E8%81%8C%E4%BD%8D%E6%8E%A8%E8%8D%90/">职位推荐</a><span class="category-list-count">2</span></li>
                </ul>
              </div>
            </div>
            <div class="sidebar-panel sidebar-panel-friends sidebar-panel-active">
              <h4 class="name"> 友情链接 </h4>
              <ul class="friends">
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/j2dub.jpg">
                  </span>
                  <span class="link">
                    <a href="https://www.findhao.net/" target="_blank" rel="noopener">FindHao</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/ou6mm.jpg">
                  </span>
                  <span class="link">
                    <a href="https://diygod.me/" target="_blank" rel="noopener">DIYgod</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/6apxu.jpg">
                  </span>
                  <span class="link">
                    <a href="https://www.51dev.com/" target="_blank" rel="noopener">IT技术社区</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://www.jankl.com/img/titleshu.jpg">
                  </span>
                  <span class="link">
                    <a href="https://www.jankl.com/" target="_blank" rel="noopener">liberalist</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/bqlbs.png">
                  </span>
                  <span class="link">
                    <a href="http://www.urselect.com/" target="_blank" rel="noopener">优社电商</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/8s88c.jpg">
                  </span>
                  <span class="link">
                    <a href="https://www.yuanrenxue.com/" target="_blank" rel="noopener">猿人学</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/2wgg5.jpg">
                  </span>
                  <span class="link">
                    <a href="https://www.yunlifang.cn/" target="_blank" rel="noopener">云立方</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/shwr6.png">
                  </span>
                  <span class="link">
                    <a href="http://lanbing510.info/" target="_blank" rel="noopener">冰蓝</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/blvoh.jpg">
                  </span>
                  <span class="link">
                    <a href="https://lengyue.me/" target="_blank" rel="noopener">冷月</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="http://qianxunclub.com/favicon.png">
                  </span>
                  <span class="link">
                    <a href="http://qianxunclub.com/" target="_blank" rel="noopener">千寻啊千寻</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/0044u.jpg">
                  </span>
                  <span class="link">
                    <a href="http://kodcloud.com/" target="_blank" rel="noopener">可道云</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/ygnpn.jpg">
                  </span>
                  <span class="link">
                    <a href="http://www.kunkundashen.cn/" target="_blank" rel="noopener">坤坤大神</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/22uv1.png">
                  </span>
                  <span class="link">
                    <a href="http://www.cenchong.com/" target="_blank" rel="noopener">岑冲博客</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/ev9kl.png">
                  </span>
                  <span class="link">
                    <a href="http://www.zxiaoji.com/" target="_blank" rel="noopener">张小鸡</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://www.503error.com/favicon.ico">
                  </span>
                  <span class="link">
                    <a href="https://www.503error.com/" target="_blank" rel="noopener">张志明个人博客</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/x714o.jpg">
                  </span>
                  <span class="link">
                    <a href="http://www.hubwiz.com/" target="_blank" rel="noopener">汇智网</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/129d8.png">
                  </span>
                  <span class="link">
                    <a href="https://www.bysocket.com/" target="_blank" rel="noopener">泥瓦匠BYSocket</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://www.xiongge.club/favicon.ico">
                  </span>
                  <span class="link">
                    <a href="https://www.xiongge.club/" target="_blank" rel="noopener">熊哥club</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/3w4fe.png">
                  </span>
                  <span class="link">
                    <a href="https://zerlong.com/" target="_blank" rel="noopener">知语</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/44hxf.png">
                  </span>
                  <span class="link">
                    <a href="http://redstonewill.com/" target="_blank" rel="noopener">红色石头</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/8g1fk.jpg">
                  </span>
                  <span class="link">
                    <a href="http://www.laodong.me/" target="_blank" rel="noopener">老董博客</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/wkaus.jpg">
                  </span>
                  <span class="link">
                    <a href="https://zhaoshuai.me/" target="_blank" rel="noopener">碎念</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/pgo0r.jpg">
                  </span>
                  <span class="link">
                    <a href="https://www.chenwenguan.com/" target="_blank" rel="noopener">陈文管的博客</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/kk82a.jpg">
                  </span>
                  <span class="link">
                    <a href="https://www.lxlinux.net/" target="_blank" rel="noopener">良许Linux教程网</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/lj0t2.jpg">
                  </span>
                  <span class="link">
                    <a href="https://tanqingbo.cn/" target="_blank" rel="noopener">IT码农</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/i8cdr.png">
                  </span>
                  <span class="link">
                    <a href="https://junyiseo.com/" target="_blank" rel="noopener">均益个人博客</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/chwv2.png">
                  </span>
                  <span class="link">
                    <a href="https://brucedone.com/" target="_blank" rel="noopener">大鱼的鱼塘</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/2y43o.png">
                  </span>
                  <span class="link">
                    <a href="http://bbs.nightteam.cn/" target="_blank" rel="noopener">夜幕爬虫安全论坛</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/zvc3w.jpg">
                  </span>
                  <span class="link">
                    <a href="https://www.weishidong.com/" target="_blank" rel="noopener">韦世东的技术专栏</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/ebudy.jpg">
                  </span>
                  <span class="link">
                    <a href="https://chuanjiabing.com/" target="_blank" rel="noopener">穿甲兵技术社区</a>
                  </span>
                </li>
              </ul>
            </div>
          </div>
        </aside>
        <div id="sidebar-dimmer"></div>
      </div>
    </main>
    <footer class="footer">
      <div class="footer-inner">
        <div class="copyright"> &copy; <span itemprop="copyrightYear">2021</span>
          <span class="with-love">
            <i class="fa fa-heart"></i>
          </span>
          <span class="author" itemprop="copyrightHolder">崔庆才丨静觅</span>
          <span class="post-meta-divider">|</span>
          <span class="post-meta-item-icon">
            <i class="fa fa-chart-area"></i>
          </span>
          <span title="站点总字数">2.6m</span>
          <span class="post-meta-divider">|</span>
          <span class="post-meta-item-icon">
            <i class="fa fa-coffee"></i>
          </span>
          <span title="站点阅读时长">39:54</span>
        </div>
        <div class="powered-by">由 <a href="https://hexo.io/" class="theme-link" rel="noopener" target="_blank">Hexo</a> & <a href="https://pisces.theme-next.org/" class="theme-link" rel="noopener" target="_blank">NexT.Pisces</a> 强力驱动 </div>
        <div class="beian"><a href="https://beian.miit.gov.cn/" rel="noopener" target="_blank">京ICP备18015597号-1 </a>
        </div>
        <script>
          (function ()
          {
            function leancloudSelector(url)
            {
              url = encodeURI(url);
              return document.getElementById(url).querySelector('.leancloud-visitors-count');
            }

            function addCount(Counter)
            {
              var visitors = document.querySelector('.leancloud_visitors');
              var url = decodeURI(visitors.id);
              var title = visitors.dataset.flagTitle;
              Counter('get', '/classes/Counter?where=' + encodeURIComponent(JSON.stringify(
              {
                url
              }))).then(response => response.json()).then((
              {
                results
              }) =>
              {
                if (results.length > 0)
                {
                  var counter = results[0];
                  leancloudSelector(url).innerText = counter.time + 1;
                  Counter('put', '/classes/Counter/' + counter.objectId,
                  {
                    time:
                    {
                      '__op': 'Increment',
                      'amount': 1
                    }
                  }).catch(error =>
                  {
                    console.error('Failed to save visitor count', error);
                  });
                }
                else
                {
                  Counter('post', '/classes/Counter',
                  {
                    title,
                    url,
                    time: 1
                  }).then(response => response.json()).then(() =>
                  {
                    leancloudSelector(url).innerText = 1;
                  }).catch(error =>
                  {
                    console.error('Failed to create', error);
                  });
                }
              }).catch(error =>
              {
                console.error('LeanCloud Counter Error', error);
              });
            }

            function showTime(Counter)
            {
              var visitors = document.querySelectorAll('.leancloud_visitors');
              var entries = [...visitors].map(element =>
              {
                return decodeURI(element.id);
              });
              Counter('get', '/classes/Counter?where=' + encodeURIComponent(JSON.stringify(
              {
                url:
                {
                  '$in': entries
                }
              }))).then(response => response.json()).then((
              {
                results
              }) =>
              {
                for (let url of entries)
                {
                  let target = results.find(item => item.url === url);
                  leancloudSelector(url).innerText = target ? target.time : 0;
                }
              }).catch(error =>
              {
                console.error('LeanCloud Counter Error', error);
              });
            }
            let
            {
              app_id,
              app_key,
              server_url
            } = {
              "enable": true,
              "app_id": "6X5dRQ0pnPWJgYy8SXOg0uID-gzGzoHsz",
              "app_key": "ziLDVEy73ne5HtFTiGstzHMS",
              "server_url": "https://6x5drq0p.lc-cn-n1-shared.com",
              "security": false
            };

            function fetchData(api_server)
            {
              var Counter = (method, url, data) =>
              {
                return fetch(`${api_server}/1.1${url}`,
                {
                  method,
                  headers:
                  {
                    'X-LC-Id': app_id,
                    'X-LC-Key': app_key,
                    'Content-Type': 'application/json',
                  },
                  body: JSON.stringify(data)
                });
              };
              if (CONFIG.page.isPost)
              {
                if (CONFIG.hostname !== location.hostname) return;
                addCount(Counter);
              }
              else if (document.querySelectorAll('.post-title-link').length >= 1)
              {
                showTime(Counter);
              }
            }
            let api_server = app_id.slice(-9) !== '-MdYXbMMI' ? server_url : `https://${app_id.slice(0, 8).toLowerCase()}.api.lncldglobal.com`;
            if (api_server)
            {
              fetchData(api_server);
            }
            else
            {
              fetch('https://app-router.leancloud.cn/2/route?appId=' + app_id).then(response => response.json()).then((
              {
                api_server
              }) =>
              {
                fetchData('https://' + api_server);
              });
            }
          })();

        </script>
      </div>
      <div class="footer-stat">
        <span id="cnzz_stat_icon_1279355174"></span>
        <script type="text/javascript">
          document.write(unescape("%3Cspan id='cnzz_stat_icon_1279355174'%3E%3C/span%3E%3Cscript src='https://v1.cnzz.com/z_stat.php%3Fid%3D1279355174%26online%3D1%26show%3Dline' type='text/javascript'%3E%3C/script%3E"));

        </script>
      </div>
    </footer>
  </div>
  <script src="//cdn.jsdelivr.net/npm/animejs@3.2.1/lib/anime.min.js"></script>
  <script src="//cdn.jsdelivr.net/npm/pangu@4/dist/browser/pangu.min.js"></script>
  <script src="/js/utils.js"></script>
  <script src="/.js"></script>
  <script src="/js/schemes/pisces.js"></script>
  <script src="/.js"></script>
  <script src="/js/next-boot.js"></script>
  <script src="/.js"></script>
  <script>
    (function ()
    {
      var canonicalURL, curProtocol;
      //Get the <link> tag
      var x = document.getElementsByTagName("link");
      //Find the last canonical URL
      if (x.length > 0)
      {
        for (i = 0; i < x.length; i++)
        {
          if (x[i].rel.toLowerCase() == 'canonical' && x[i].href)
          {
            canonicalURL = x[i].href;
          }
        }
      }
      //Get protocol
      if (!canonicalURL)
      {
        curProtocol = window.location.protocol.split(':')[0];
      }
      else
      {
        curProtocol = canonicalURL.split(':')[0];
      }
      //Get current URL if the canonical URL does not exist
      if (!canonicalURL) canonicalURL = window.location.href;
      //Assign script content. Replace current URL with the canonical URL
      ! function ()
      {
        var e = /([http|https]:\/\/[a-zA-Z0-9\_\.]+\.baidu\.com)/gi,
          r = canonicalURL,
          t = document.referrer;
        if (!e.test(r))
        {
          var n = (String(curProtocol).toLowerCase() === 'https') ? "https://sp0.baidu.com/9_Q4simg2RQJ8t7jm9iCKT-xh_/s.gif" : "//api.share.baidu.com/s.gif";
          t ? (n += "?r=" + encodeURIComponent(document.referrer), r && (n += "&l=" + r)) : r && (n += "?l=" + r);
          var i = new Image;
          i.src = n
        }
      }(window);
    })();

  </script>
  <script src="/js/local-search.js"></script>
  <script src="/.js"></script>
  <link rel="stylesheet" href="//cdn.jsdelivr.net/npm/gitalk@1/dist/gitalk.min.css">
  <script>
    NexT.utils.loadComments(document.querySelector('#gitalk-container'), () =>
    {
      NexT.utils.getScript('//cdn.jsdelivr.net/npm/gitalk@1/dist/gitalk.min.js', () =>
      {
        var gitalk = new Gitalk(
        {
          clientID: '4c86ce1d7c4fbb3b277c',
          clientSecret: '4927beb0f90e2c07e66c99d9d2529cf3eb8ac8e4',
          repo: 'Blog',
          owner: 'germey',
          admin: ['germey'],
          id: 'bf59fb2c5ab8791b313b7a78dd124769',
          language: 'zh-CN',
          distractionFreeMode: true
        });
        gitalk.render('gitalk-container');
      }, window.Gitalk);
    });

  </script>
</body>

</html>
